
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import cpi
import collections
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split


df = pd.read_csv("IMDb-movies.csv")

/home/alex/.miniconda/lib/python3.8/site-packages/IPython/core/interactiveshell.py:3165: DtypeWarning: Columns (3) have mixed types.Specify dtype option on import or set low_memory=False.
  has_raised = await self.run_ast_nodes(code_ast.body, cell_name,


pd.set_option('display.max_columns', None)


df.head()


df.dtypes

imdb_title_id             object
title                     object
original_title            object
year                      object
date_published            object
genre                     object
duration                   int64
country                   object
language                  object
director                  object
writer                    object
production_company        object
actors                    object
description               object
avg_vote                 float64
votes                      int64
budget                    object
usa_gross_income          object
worlwide_gross_income     object
metascore                float64
reviews_from_users       float64
reviews_from_critics     float64
dtype: object


years = df['year'].tolist()

print("Non-numeric year values:")

for i in range(len(years)):
    if not str(years[i]).isdigit():
        print("Row " + str(i) + ": " + str(years[i]))

Non-numeric year values:
Row 83917: TV Movie 2019


#First, we cast our year column - which is currently a Series object - to a Series of strings.
#(A Series is a Pandas datatype that's similar to a list, but has extra functionality.) 
#Then we use the apply() function to perform a certain function on each value in the Series and stores the resulting value of that function.
#The function we are applying takes all of the numeric characters of a given string and returns them as a string. 

df['year'] = df['year'].astype(str).apply(lambda x: ''.join(i for i in x if i.isdigit()))

#setting the year column as a column of ints
df['year'] = df['year'].astype(int)


df = df.rename(columns={'worlwide_gross_income': 'income'})


df = df[['year', 'genre', 'duration', 'country', 'budget', 'income']]


#removing dollar signs:
df['budget'] = df['budget'].str.replace('\$', '', regex=True)
df['income'] = df['income'].str.replace('\$', '', regex=True)

#If a value is in a currency other than USD, the currency symbol will still show up.
#To keep things easy, let's just set all remaining non-numeric values to NaN using the following lines:
df['budget'] = df['budget'].apply(lambda x: pd.to_numeric(x, errors='coerce'))
df['income'] = df['income'].apply(lambda x: pd.to_numeric(x, errors='coerce'))

#setting the columns to floats:
df['budget'] = df['budget'].astype(float)
#df['income'] = df['income'].astype(float)


df = df.dropna(subset=['income'])
df = df.reset_index().drop(['index'], axis=1)


df.head()


#first we group the rows by their year, then for each grouping, we take the mean of that grouping's income
#this gives us a dataframe containing the mean income for every year
df_by_year = df.groupby(['year'])['income'].mean().reset_index()

#next need to make a Matplotlib figure to plot the data on, then plot our data on that figure
fig, ax = plt.subplots()
x = df_by_year['year']
y = df_by_year['income']
ax.plot(x, y)

#always title and label your plots so you know what you're looking at
ax.set_title("Average worldwide gross income for movies over time\n")
ax.set_xlabel("Year")
ax.set_ylabel("Average Gross Income\n(in tens of millions of USD)")

fig.show()


#here we make a list of each income values adjusted for inflation
#we use a try/except because the inflation function will throw an error if it doesn't have inflation values for a given year
lst = []
for index, row in df.iterrows():
    try:
        lst.append(cpi.inflate(row['income'], row['year']))
    except:
        lst.append(np.NaN)

df['adj_income'] = lst

#now we generate and plot the means just like before, but with adjusted income
df_by_year = df.groupby(['year'])['adj_income'].mean().reset_index()

fig, ax = plt.subplots()
x = df_by_year['year']
y = df_by_year['adj_income']
ax.plot(x, y)

#always title and label your plots so you know what you're looking at
ax.set_title("Average worldwide gross income for movies over time, adjusted for inflation\n")
ax.set_xlabel("Year")
ax.set_ylabel("Average Gross Income\n(in hundreds of millions of USD)")

fig.show()


fig, ax = plt.subplots()

#Seaborn can make plots very easily
ax = sns.regplot(data=df, x='budget', y='income', truncate=False, line_kws={"color": "red"})

ax.set_title("Budget vs Worldwide Gross Income\n")
ax.set_xlabel("Budget (in hundreds of millions of USD)")
ax.set_ylabel("Gross Income (in billions of USD)")

fig.show()


fig, ax = plt.subplots()

ax = sns.regplot(data=df, x='duration', y='income', truncate=False, line_kws={"color": "red"})

ax.set_title("Duration vs Worldwide Gross Income\n")
ax.set_xlabel("Duration (minutes)")
ax.set_ylabel("Gross Income (in billions of USD)")

fig.show()


#we make a list containing every instance of a country name from the country column of our dataframe

#creating a list of lists of country names by splitting on commas
con_list = list(df['country'].dropna().str.split(','))
#flattening our list of lists
con_list = [item for sublist in con_list for item in sublist]
#removing whitespace in case the names have irregular whitespace
con_list = [x.strip() for x in con_list]

#we create a dictionary giving a frequency count for each country name, then turn that dictionary into a dataframe so that we can sort by frequency
freq = pd.DataFrame.from_dict(collections.Counter(con_list), orient='index', columns=['frequency'])
freq = freq.sort_values(by=['frequency'], ascending=False)

freq.head(10)


#removing all irregular spacing
df['country'] = df['country'].str.replace(' ', '')

#the get_dummies function allows us to easily make our country columns
df_con = df['country'].str.get_dummies(sep=',')
#we only need the countries from the top 5
df_con = df_con[['USA', 'France', 'UK', 'Germany', 'India']]

#joining our country columns to the main df
df = df.join(df_con)

#we won't need the original country column going forward
df = df.drop(['country'], axis=1)


df.head()


#getting the movies for each country
#con_dfs = {}
#for i in ['USA', 'France', 'UK', 'Germany', 'India']:
#    con_dfs[i] = df.loc[df[i] == 1][['year', 'income']]

#our x value will be our list of countries
#x = list(con_dfs.keys())

x = ['USA', 'France', 'UK', 'Germany', 'India']
#our y value will be the mean income for each country
#y = [i['income'].mean() for i in list(con_dfs.values())]

y = []
for i in x:
    df_ = df.loc[df[i] == 1]
    y.append(df_['income'].mean())

#sorting our x and y values by the y values, so that our bars will be in order from shortest to tallest 
tuples = zip(*sorted(zip(y, x)))
y, x = [list(i) for i in  tuples]

fig, ax = plt.subplots()

ax.bar(x, y)

ax.set_title("Average Worldwide Gross Income of Movies by Country\n")
ax.set_ylabel("Average Gross Income\n(in tens of millions of USD)")

fig.show()


#generating the genre columns

df['genre'] = df['genre'].str.replace(' ', '')
df_genres = df['genre'].str.get_dummies(sep=',')
df = df.join(df_genres)
df = df.drop(['genre'], axis=1)

#storing the list of genres:
genres = list(df_genres.columns)

#getting our x and y
#genre_dfs = {}
#for i in genres:
#    genre_dfs[i] = df.loc[df[i] == 1][['year', 'income']]
x = genres

y = []
for i in x:
    df_ = df.loc[df[i] == 1]
    y.append(df_['income'].mean())

#y = [i['income'].mean() for i in list(genre_dfs.values())]
tuples = zip(*sorted(zip(y, x)))
y, x = [list(i) for i in  tuples]

#plotting
fig, ax = plt.subplots(figsize=(10, 10))
#horizontal bar graphs are better if your graph has a lot of bars
ax.barh(x, y)
ax.set_title("Average Worldwide Gross Income of Movies by Genre")
ax.set_xlabel("Average Gross Income (in hundreds of millions of USD)")
fig.show()


print("Dataframe number of rows: " + str(df.shape[0]))
print("\nNumber of NaN values per column:")
print(df.isna().sum())

Dataframe number of rows: 30955

Number of NaN values per column:
year               0
duration           0
budget         21930
income             0
adj_income         0
USA                0
France             0
UK                 0
Germany            0
India              0
Action             0
Adventure          0
Animation          0
Biography          0
Comedy             0
Crime              0
Documentary        0
Drama              0
Family             0
Fantasy            0
Film-Noir          0
History            0
Horror             0
Music              0
Musical            0
Mystery            0
Romance            0
Sci-Fi             0
Sport              0
Thriller           0
War                0
Western            0
dtype: int64


df_bug = df.dropna(subset=['budget'])
df_bug = df_bug.reset_index().drop(['index'], axis=1)
#df_bug[['income', 'budget']] = RobustScaler().fit_transform(df_bug[['income', 'budget']])


#first let's get our X variables (input variables) and Y variable (output variable)
#if your X only contain one variable, you will convert it to an np.array and reshape it using np.array.reshape(-1, 1)
X = np.array(df_bug['budget']).reshape(-1, 1)
Y = df_bug['income']

#next let's divide our X and Y data into a testing set and a training set
#we'll have 25% of our data in the test set, which is the standard proportion
#having a specific seed for randomization (random_state) means our results are replicable
Xtrain, Xtest, Ytrain, Ytest = train_test_split(X, Y, test_size = 0.25, random_state = 35)

#now let's make a regression object and train it on our training set
reg = LinearRegression()
reg.fit(Xtrain, Ytrain)

#let's take a look at what our equation looks like
print("Linear Regression Coefficients:\n" + str(reg.coef_) + "\n")
print("Linear Regression Intercept:\n" + str(reg.intercept_) + "\n")
print("Linear Regression Equation (x is budget, y is income):\ny = " + str(reg.coef_[0]) + ' * x + ' + str(reg.intercept_) + "\n")

#now let's get our r-squared values using the score() function
acc_train = reg.score(Xtrain, Ytrain)
acc_test = reg.score(Xtest, Ytest)


#let's see what our accuracies look like
print("Linear Regression Accuracy (R-squared) for Training Data: " + str(acc_train))
print("Linear Regression Accuracy (R-squared) for Test Data: " + str(acc_test))
print("\n")

#and let's plot our regression
fig, ax = plt.subplots()
ax.scatter(df_bug['budget'], df_bug['income'])
ax.plot(X, reg.predict(X), color='red')
ax.set_title("Linear Regression for Budget vs Income")
ax.set_xlabel("Budget")
ax.set_ylabel("Income")
fig.show()

Linear Regression Coefficients:
[3.27406362]

Linear Regression Intercept:
-15229062.394591153

Linear Regression Equation (x is budget, y is income):
y = 3.2740636202127504 * x + -15229062.394591153

Linear Regression Accuracy (R-squared) for Training Data: 0.5666269492892935
Linear Regression Accuracy (R-squared) for Test Data: 0.5591197402272339


#generating our residuals
res = Y - reg.predict(X)

#plotting them
fig, ax = plt.subplots()
ax.scatter(X, res)

#plotting a trend line for our residuals
res_trend = LinearRegression()
res_trend.fit(X, res)
ax.plot(X, res_trend.predict(X), color='red')
trend = "y = " + str(res_trend.coef_[0]) + " * x + " + str(res_trend.intercept_)

#trend = np.polyfit(X[0], res, 1)
#trendline = np.poly1d(trend)
#ax.plot(X, trendline(X[0]), c='red')

ax.set_title("Residuals for Linear Regression for Budget vs Income\n")
ax.set_xlabel("Budget\n\nTrend line: " + trend)
ax.set_ylabel("Residuals")
fig.show()


def gen_LinReg(df, xcols, ycol, formula=False, r2=False):
    if not (formula or r2):
        return

    print("Linear Regression")
    print("X = " + str(xcols))
    print("Y = " + str(ycol) + "\n")

    #getting X and Y
    X = df[xcols]
    #reshaping if X is one column
    if len(xcols) == 1:
        X = np.array(X).reshape(-1, 1)
    Y = df[ycol]

    #splitting data into training and test sets
    Xtrain, Xtest, Ytrain, Ytest = train_test_split(X, Y, test_size = 0.2, random_state = 35)

    #generating regression
    reg = LinearRegression()
    reg.fit(Xtrain, Ytrain)

    #viewing regression
    if formula:
        print("Coefficients:\n" + str(reg.coef_))
        print("\nIntercept:\n" + str(reg.intercept_) + "\n")

    #viewing r2
    if r2:
        #getting r2
        acc_train = reg.score(Xtrain, Ytrain)
        acc_test = reg.score(Xtest, Ytest)

        print("Accuracy (R-squared) for Training Data: " + str(acc_train))
        print("Accuracy (R-squared) for Test Data: " + str(acc_test))


gen_LinReg(df_bug, ['year', 'duration', 'budget'], 'income', formula=True, r2=True)

Linear Regression
X = ['year', 'duration', 'budget']
Y = income

Coefficients:
[-3.09183340e+04  3.39113246e+05  3.20746419e+00]

Intercept:
12189434.839708388

Accuracy (R-squared) for Training Data: 0.5659868927316661
Accuracy (R-squared) for Test Data: 0.5712998701736798


gen_LinReg(df_bug, ['year'], 'income', r2=True)

Linear Regression
X = ['year']
Y = income

Accuracy (R-squared) for Training Data: 0.015503387320112139
Accuracy (R-squared) for Test Data: 0.010664703761729788


gen_LinReg(df_bug, ['duration'], 'income', r2=True)

Linear Regression
X = ['duration']
Y = income

Accuracy (R-squared) for Training Data: 0.06321684125890437
Accuracy (R-squared) for Test Data: 0.06194652490740049


gen_LinReg(df, ['USA', 'France', 'UK', 'Germany', 'India'], 'income', r2=True)

Linear Regression
X = ['USA', 'France', 'UK', 'Germany', 'India']
Y = income

Accuracy (R-squared) for Training Data: 0.06759725355273039
Accuracy (R-squared) for Test Data: 0.06528237344772114


gen_LinReg(df, genres, 'income', r2=True)

Linear Regression
X = ['Action', 'Adventure', 'Animation', 'Biography', 'Comedy', 'Crime', 'Documentary', 'Drama', 'Family', 'Fantasy', 'Film-Noir', 'History', 'Horror', 'Music', 'Musical', 'Mystery', 'Romance', 'Sci-Fi', 'Sport', 'Thriller', 'War', 'Western']
Y = income

Accuracy (R-squared) for Training Data: 0.11696974116788017
Accuracy (R-squared) for Test Data: 0.09389917778136592


gen_LinReg(df_bug, ['year', 'duration', 'budget', 'USA', 'France', 'UK', 'Germany', 'India'] + genres, 'income', r2=True)

Linear Regression
X = ['year', 'duration', 'budget', 'USA', 'France', 'UK', 'Germany', 'India', 'Action', 'Adventure', 'Animation', 'Biography', 'Comedy', 'Crime', 'Documentary', 'Drama', 'Family', 'Fantasy', 'Film-Noir', 'History', 'Horror', 'Music', 'Musical', 'Mystery', 'Romance', 'Sci-Fi', 'Sport', 'Thriller', 'War', 'Western']
Y = income

Accuracy (R-squared) for Training Data: 0.5781010972286499
Accuracy (R-squared) for Test Data: 0.5856436989159083

	imdb_title_id	title	original_title	year	date_published	genre	duration	country	language	director	writer	production_company	actors	description	avg_vote	votes	budget	usa_gross_income	worlwide_gross_income	metascore	reviews_from_users	reviews_from_critics
0	tt0000009	Miss Jerry	Miss Jerry	1894	1894-10-09	Romance	45	USA	None	Alexander Black	Alexander Black	Alexander Black Photoplays	Blanche Bayliss, William Courtenay, Chauncey D...	The adventures of a female reporter in the 1890s.	5.9	154	NaN	NaN	NaN	NaN	1.0	2.0
1	tt0000574	The Story of the Kelly Gang	The Story of the Kelly Gang	1906	1906-12-26	Biography, Crime, Drama	70	Australia	None	Charles Tait	Charles Tait	J. and N. Tait	Elizabeth Tait, John Tait, Norman Campbell, Be...	True story of notorious Australian outlaw Ned ...	6.1	589	$ 2250	NaN	NaN	NaN	7.0	7.0
2	tt0001892	Den sorte drøm	Den sorte drøm	1911	1911-08-19	Drama	53	Germany, Denmark	NaN	Urban Gad	Urban Gad, Gebhard Schätzler-Perasini	Fotorama	Asta Nielsen, Valdemar Psilander, Gunnar Helse...	Two men of high rank are both wooing the beaut...	5.8	188	NaN	NaN	NaN	NaN	5.0	2.0
3	tt0002101	Cleopatra	Cleopatra	1912	1912-11-13	Drama, History	100	USA	English	Charles L. Gaskill	Victorien Sardou	Helen Gardner Picture Players	Helen Gardner, Pearl Sindelar, Miss Fielding, ...	The fabled queen of Egypt's affair with Roman ...	5.2	446	$ 45000	NaN	NaN	NaN	25.0	3.0
4	tt0002130	L'Inferno	L'Inferno	1911	1911-03-06	Adventure, Drama, Fantasy	68	Italy	Italian	Francesco Bertolini, Adolfo Padovan	Dante Alighieri	Milano Film	Salvatore Papa, Arturo Pirovano, Giuseppe de L...	Loosely adapted from Dante's Divine Comedy and...	7.0	2237	NaN	NaN	NaN	NaN	31.0	14.0

	year	genre	duration	country	budget	income
0	1916	Drama, Fantasy, Horror	63	Russia	NaN	144968.0
1	1920	Fantasy, Horror, Mystery	76	Germany	18000.0	8811.0
2	1921	Drama	107	Norway	NaN	4272.0
3	1920	Comedy, Drama, Romance	75	USA	NaN	772155.0
4	1921	Drama, Romance, War	150	USA	800000.0	9183673.0

	frequency
USA	11339
France	4307
UK	3131
Germany	2252
India	1936
Japan	1610
Italy	1533
Canada	1380
Spain	1340
South Korea	976

	year	genre	duration	budget	income	adj_income	USA	Germany
0	1916	Drama, Fantasy, Horror	63	NaN	144968.0	3.442139e+06	0	0
1	1920	Fantasy, Horror, Mystery	76	18000.0	8811.0	1.140192e+05	0	1
2	1921	Drama	107	NaN	4272.0	6.176763e+04	0	0
3	1920	Comedy, Drama, Romance	75	NaN	772155.0	9.992110e+06	1	0
4	1921	Drama, Romance, War	150	800000.0	9183673.0	1.327841e+08	1	0

Analyzing IMDb's Movie Database: A Data Science Tutorial¶

By Alexander Melville¶

Introduction¶

About IMDb¶

Data Source¶

Libraries¶

Part 1: Data Processing¶

Loading the dataset¶

A warning?¶

Viewing the data¶

Tidying the data¶

Part 2: Exploratory Analysis & Data Visualization¶

Year vs Income¶

Budget vs Income¶

Duration vs Income¶

Country vs Income¶

Genre vs Income¶

Part 3: Analysis Using Machine Learning¶

Understanding Linear Regressions¶

Testing the Effectiveness of Linear Regressions¶

More Tidying of Data¶

Generating Simple Linear Regressions¶

Residual Plots¶

Generating Complex Linear Regressions¶

A Regression for Year, Budget, and Duration vs Income¶

Regressions Based on Year and on Duration¶

A Regression for Countries vs Income¶

A Regression for Genres vs Income¶

A Regression for All Factors vs Income¶

Part 4: Conclusion¶

Recap¶

Going Forward¶